/*
* @Author: steven.wang
* @Date:   2019-03-20 16:49:03
* @Last Modified by:   steven.wang
* @Last Modified time: 2019-03-23 02:24:05
*/
const fs = require("fs");
const cheerio = require('cheerio');
const asyncQuene = require("async").queue;
const request = require('superagent');
require('superagent-charset')(request);
const sleep = require('sleep');
const tools = require('./tools');
const config = require('./config');

/**
 * 获取所有job的链接
 * @author steven.wang
 * @DateTime 2019-03-23T00:57:37+0800
 */
const getJobUrls = () => {
    return new Promise((resolve, reject) => {
        console.log('Start get jobs .....');
        let jobs = [], errors = [], count = 0;
        let q = asyncQuene(async (pageUrl, cb) => {
            if(count%100 == 1) {
                sleep.sleep(5);
            }
            count++;
            try {
                let $ = await tools.getHtmlAsync(pageUrl.url);
                console.log(`download ${pageUrl.url} success`);
                $('.job-list>ul>li').each(function(idx, elem) {
                    jobs.push({
                        type: pageUrl.name,
                        name: $(this).find('.job-title').text(),
                        salary: $(this).find('.red').text(),
                        company: $(this).find('.company-text>.name').text(),
                        url: 'https://www.zhipin.com' + $(this).find('.info-primary>.name>a').attr('href'),
                    });
                });
            } catch (err) {
                errors.push(pageUrl);
                console.log(`Error : get job list - download ${pageUrl.url} err : ${err}`);
            } finally {
                cb();
            }
        }, config.concurrent);
        /**
         * 监听：当所有任务都执行完以后，将调用该函数
         */
        q.drain = () => {
            if (errors.length > 0) {
                console.log('Get job url err: ' + JSON.stringify(errors));
                tools.writeJsonToFile('files/boss-job-url-error.json', errors);
            }
            console.log('Get job url complete');
            resolve(jobs);
        }

        let pageUrls = [];
        for (let item of config.boss.list) {
            for (let subItem of item.children) {
                for (let i = subItem.startPage; i <= subItem.endPage; i++) {
                    pageUrls.push({
                        name: subItem.name,
                        url: subItem.url + `?period=5&page=${i}&ka=page-${i}`
                    });
                }
            }
        }
        q.push(pageUrls);
    });
}
/**
 * 获取所有job的详情
 * @author steven.wang
 * @DateTime 2019-03-23T00:57:59+0800
 */
const getJobsDetail = data => {
    return new Promise((resolve, reject) => {
        console.log('Start get jobs .....');
        let jobs = [], errors = [], count = 0;
        let q = asyncQuene(async (pageUrl, cb) => {
            if(count%100 == 1) {
                sleep.sleep(5);
            }
            count++;
            try {
                let $ = await tools.getHtmlAsync(pageUrl.url);
                console.log(`download ${pageUrl.url} success`);
                jobs.push({
                    source: 'BOSS直聘',
                    name: $('.job-banner>.home-inner>.detail-box>.info-primary>.name>h1').html(),
                    state: $('.job-status').html(),
                    type: pageUrl.type,
                    label: function () {
                        let result = [];
                        $('.job-tags').find('span').each(function() {
                            result.push($(this).html())
                        });
                        return result.join(',');
                    },
                    work_time: '全职',
                    publish_time: $('.sider-company>.gray').html().replace(/发布于：/g, ''),
                    salary: $('.job-banner>.home-inner>.detail-box>.info-primary>.name>.salary').html().trim(),
                    city: $('.job-banner>.home-inner>.detail-box>.info-primary>p').html().split('<em class=\"dolt\"></em>')[0],
                    experience: $('.job-banner>.home-inner>.detail-box>.info-primary>p').html().split('<em class=\"dolt\"></em>')[1],
                    education: $('.job-banner>.home-inner>.detail-box>.info-primary>p').html().split('<em class=\"dolt\"></em>')[2],
                    advantage: function () {
                        let result = [];
                        $('.job-tags').find('span').each(function() {
                            result.push($(this).html())
                        });
                        return result.join(',');
                    },
                    desc: $('.detail-content>.job-sec').eq(0).find('.text').html().trim(),
                    address: $('.location-address').html(),
                    hr_name: $('.job-box>.inner>.job-detail>.detail-op>.name').text(),
                    hr_position: $('.job-box>.inner>.job-detail>.detail-op>.gray').text().split('·')[0],
                    company_name: $('.sider-company>.company-info').text().trim(),
                    company_id: '',
                    company_detail_url: config.boss.host + $('.sider-company>.company-info>a').eq(0).attr('href')
                });
            } catch (err) {
                errors.push(pageUrl);
                console.log(`Error : get job list - download ${pageUrl.url} err : ${err}`);
            } finally {
                cb();
            }
        }, config.concurrent);
        /**
         * 监听：当所有任务都执行完以后，将调用该函数
         */
        q.drain = () => {
            if (errors.length > 0) {
                console.log('Get job detail err: ' + JSON.stringify(errors));
                tools.writeJsonToFile('files/boss-job-detail-error.json', errors);
            }
            console.log('Get job detail complete');
            resolve(jobs);
        }

        q.push(data);
    });
}

async function app() {
    let jobList = await getJobUrls();
    tools.writeJsonToFile(config.boss.jobUrlPath, jobList);

    jobList = await getJobsDetail(jobList);
    tools.writeJsonToFile(config.boss.jobDetailPath, jobList);

    // 存入数据库
    // let insertResult = await tools.insertPhoto(values);
    // console.log(insertResult)
}

app();
